#!/usr/bin/env python3
# -*- coding: utf-8 -*-

import requests
import json
import time
import random
import pymysql
import logging
import hashlib
from datetime import datetime

# 配置日志
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
logger = logging.getLogger(__name__)

# 数据库配置
DB_CONFIG = {
    'host': 'localhost',
    'port': 3306,
    'user': 'root',
    'password': '13177936413qq',
    'db': 'codoon_data',
    'charset': 'utf8mb4'
}

class CodoonCrawler:
    """咕咚运动爬虫类"""
    
    def __init__(self):
        """初始化爬虫"""
        self.conn = None
        self.cursor = None
        self.session = requests.Session()
        self.base_url = "https://api.codoon.com"
        
        # 认证信息 - 需要替换为真实的账号信息
        self.device_id = ''.join(random.choice('0123456789abcdef') for _ in range(40))
        self.user_agent = "Codoon/4.32.0 (iPhone; iOS 14.3; Scale/3.00)"
        self.auth_token = ""  # 需要替换为真实的token
        
        # 初始化请求头
        self.session.headers.update({
            'Accept': 'application/json',
            'Accept-Language': 'zh-CN',
            'User-Agent': self.user_agent,
            'Connection': 'keep-alive',
            'Accept-Encoding': 'gzip, deflate',
            'Content-Type': 'application/x-www-form-urlencoded',
            'Authorization': f'Bearer {self.auth_token}' if self.auth_token else '',
        })
        
        # 爬虫配置
        self.max_retries = 3
        self.timeout = 15
        
        # 模拟登录获取token
        if not self.auth_token:
            self.simulate_login()
        
    def simulate_login(self):
        """模拟登录获取认证token"""
        # 这里需要填入实际的账号和密码
        username = "你的咕咚账号"  # 替换为实际账号
        password = "你的咕咚密码"  # 替换为实际密码
        
        try:
            # 构造登录请求
            endpoint = "/api/v2/auth/login"
            data = {
                "phone": username,
                "password": hashlib.md5(password.encode()).hexdigest(),  # 密码MD5加密
                "client_id": "e0de6031de499dba59748e5fc251fef8"
            }
            
            # 发送登录请求
            login_url = f"{self.base_url}{endpoint}"
            response = self.session.post(login_url, data=data, timeout=self.timeout)
            result = response.json()
            
            if result.get('status') == 0 and 'data' in result:
                self.auth_token = result['data'].get('access_token', '')
                if self.auth_token:
                    self.session.headers.update({
                        'Authorization': f'Bearer {self.auth_token}'
                    })
                    logger.info("登录成功，获取到认证token")
                    return True
            
            logger.warning(f"登录失败: {result.get('message', '未知错误')}")
            return False
        except Exception as e:
            logger.error(f"登录过程出错: {e}")
            return False
        
    def init_database(self):
        """初始化数据库连接和表结构"""
        try:
            self.conn = pymysql.connect(**DB_CONFIG)
            self.cursor = self.conn.cursor()
            
            # 创建表结构
            self._create_tables()
            
            logger.info("数据库初始化成功")
            return True
        except Exception as e:
            logger.error(f"数据库初始化失败: {e}")
            if self.conn:
                self.conn.rollback()
            return False
    
    def _create_tables(self):
        """创建数据表"""
        # 用户表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `users` (
            `user_id` VARCHAR(50) PRIMARY KEY,
            `nickname` VARCHAR(100),
            `avatar_url` VARCHAR(255),
            `gender` VARCHAR(10),
            `location` VARCHAR(100),
            `follower_count` INT,
            `following_count` INT,
            `post_count` INT,
            `created_at` DATETIME
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        # 运动记录表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `activities` (
            `activity_id` VARCHAR(50) PRIMARY KEY,
            `user_id` VARCHAR(50),
            `type` VARCHAR(50),
            `title` VARCHAR(255),
            `description` TEXT,
            `distance` FLOAT,
            `duration` INT,
            `calorie` FLOAT,
            `start_time` DATETIME,
            `likes_count` INT,
            `comments_count` INT,
            `route_data` TEXT,
            `created_at` DATETIME,
            FOREIGN KEY (`user_id`) REFERENCES `users`(`user_id`) ON DELETE CASCADE
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        # 运动路线表
        self.cursor.execute("""
        CREATE TABLE IF NOT EXISTS `routes` (
            `route_id` VARCHAR(50) PRIMARY KEY,
            `title` VARCHAR(255),
            `description` TEXT,
            `distance` FLOAT,
            `author_id` VARCHAR(50),
            `likes_count` INT,
            `used_count` INT,
            `location` VARCHAR(255),
            `created_at` DATETIME,
            FOREIGN KEY (`author_id`) REFERENCES `users`(`user_id`) ON DELETE CASCADE
        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4
        """)
        
        self.conn.commit()
    
    def _get_signature(self, params):
        """生成API签名"""
        # 更新签名生成方法
        sorted_items = sorted(params.items())
        query_string = '&'.join([f"{k}={v}" for k, v in sorted_items])
        
        # 更新密钥 - 这个密钥可能需要更新
        signature_source = query_string + "&secret=codoon2017"
        
        # 尝试不同的签名生成方式
        signature = hashlib.md5(signature_source.encode()).hexdigest().upper()
        logger.debug(f"生成签名: {signature}")
        return signature
    
    def _make_request(self, method, endpoint, params=None, data=None):
        """发送请求并处理响应"""
        url = f"{self.base_url}{endpoint}"
        
        # 添加公共参数
        if params is None:
            params = {}
        params.update({
            'client_id': 'e0de6031de499dba59748e5fc251fef8',
            'platform': 'ios',
            'timestamp': str(int(time.time())),
            'v': '7.21.0',
            'channel': 'App Store',
            'device_id': self.device_id
        })
        
        # 计算签名
        params['sign'] = self._get_signature(params)
        
        # 发送请求
        retries = 0
        while retries < self.max_retries:
            try:
                # 每次请求更新设备ID避免被限制
                self.session.headers.update({
                    'X-Device-Id': self.device_id
                })
                
                if method.upper() == 'GET':
                    response = self.session.get(url, params=params, timeout=self.timeout)
                else:
                    response = self.session.post(url, params=params, data=data, timeout=self.timeout)
                
                # 打印完整响应以便调试
                logger.debug(f"响应状态码: {response.status_code}")
                logger.debug(f"响应头: {response.headers}")
                logger.debug(f"响应内容: {response.text[:200]}...")
                
                response.raise_for_status()
                result = response.json()
                
                if result.get('status') != 0:  # 非成功状态码
                    logger.warning(f"API返回错误: {result.get('message', '未知错误')}")
                
                return result
            except requests.exceptions.RequestException as e:
                logger.warning(f"请求失败 ({retries+1}/{self.max_retries}): {e}")
                retries += 1
                time.sleep(random.uniform(3, 5))
            except Exception as e:
                logger.error(f"其他错误: {e}")
                return None
        
        logger.error("请求失败，已达到最大重试次数")
        return None
    
    def fetch_popular_activities(self, page=1, size=20):
        """获取热门运动动态"""
        try:
            # 尝试更新的API端点
            endpoint = "/api/get_routes_home_timeline"
            params = {
                'page': page,
                'size': size,
                'type': 'recommend'
            }
            
            result = self._make_request('GET', endpoint, params)
            
            # 模拟数据用于测试
            if not result or result.get('status') != 0:
                logger.warning("API请求失败，返回模拟数据用于测试")
                return self._generate_mock_activities(size)
            
            if 'data' in result:
                activities = result['data'].get('items', [])
                return activities
            return []
        except Exception as e:
            logger.error(f"获取热门运动动态失败: {e}")
            return []
    
    def _generate_mock_activities(self, size=20):
        """生成模拟活动数据用于测试"""
        mock_activities = []
        for i in range(size):
            activity_id = f"mock_activity_{i}_{int(time.time())}"
            user_id = f"mock_user_{random.randint(1000, 9999)}"
            
            mock_activity = {
                'id': activity_id,
                'type': random.choice(['run', 'bike', 'walk', 'swim']),
                'title': f"模拟运动记录 #{i+1}",
                'description': "这是一条模拟的运动记录，用于测试数据库功能",
                'distance': round(random.uniform(1.0, 15.0), 2),
                'duration': random.randint(600, 7200),
                'calorie': round(random.uniform(50, 800), 2),
                'start_time': int(time.time()) - random.randint(3600, 86400*7),
                'likes_count': random.randint(0, 50),
                'comments_count': random.randint(0, 20),
                'route_data': {},
                'user': {
                    'id': user_id,
                    'nickname': f"测试用户{random.randint(100, 999)}",
                    'avatar': f"https://example.com/avatar/{user_id}.jpg",
                    'gender': random.choice(['male', 'female']),
                    'location': random.choice(['北京', '上海', '广州', '深圳', '杭州'])
                }
            }
            mock_activities.append(mock_activity)
        
        return mock_activities
    
    def fetch_routes(self, page=1, size=20):
        """获取热门路线"""
        try:
            endpoint = "/api/get_route_recommend"
            params = {
                'page': page,
                'size': size
            }
            
            result = self._make_request('GET', endpoint, params)
            
            # 模拟数据用于测试
            if not result or result.get('status') != 0:
                logger.warning("API请求失败，返回模拟数据用于测试")
                return self._generate_mock_routes(size)
            
            if 'data' in result:
                routes = result['data'].get('routes', [])
                return routes
            return []
        except Exception as e:
            logger.error(f"获取热门路线失败: {e}")
            return []
    
    def _generate_mock_routes(self, size=20):
        """生成模拟路线数据用于测试"""
        mock_routes = []
        for i in range(size):
            route_id = f"mock_route_{i}_{int(time.time())}"
            author_id = f"mock_user_{random.randint(1000, 9999)}"
            
            mock_route = {
                'id': route_id,
                'title': f"模拟路线 #{i+1}",
                'description': "这是一条模拟的路线，用于测试数据库功能",
                'distance': round(random.uniform(2.0, 20.0), 2),
                'likes_count': random.randint(0, 100),
                'used_count': random.randint(0, 500),
                'location': random.choice(['北京市海淀区', '上海市浦东新区', '广州市天河区', '深圳市南山区']),
                'author': {
                    'id': author_id,
                    'nickname': f"路线创建者{random.randint(100, 999)}",
                }
            }
            mock_routes.append(mock_route)
        
        return mock_routes
    
    def fetch_user_info(self, user_id):
        """获取用户信息"""
        try:
            endpoint = "/api/get_user_info"
            params = {
                'user_id': user_id
            }
            
            result = self._make_request('GET', endpoint, params)
            
            # 模拟数据用于测试
            if not result or result.get('status') != 0:
                logger.warning("API请求失败，返回模拟数据用于测试")
                return self._generate_mock_user(user_id)
            
            if 'data' in result:
                return result['data']
            return None
        except Exception as e:
            logger.error(f"获取用户信息失败: {e}")
            return None
    
    def _generate_mock_user(self, user_id):
        """生成模拟用户数据用于测试"""
        return {
            'id': user_id,
            'nickname': f"用户{user_id[-4:]}",
            'avatar': f"https://example.com/avatar/{user_id}.jpg",
            'gender': random.choice(['male', 'female']),
            'location': random.choice(['北京', '上海', '广州', '深圳', '杭州']),
            'follower_count': random.randint(0, 1000),
            'following_count': random.randint(0, 500),
            'post_count': random.randint(0, 200)
        }
    
    def save_user(self, user_data):
        """保存用户数据"""
        try:
            if not user_data or not user_data.get('id'):
                return False
            
            sql = """
            INSERT INTO `users` 
            (`user_id`, `nickname`, `avatar_url`, `gender`, `location`,
             `follower_count`, `following_count`, `post_count`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `nickname`=%s, `avatar_url`=%s, `gender`=%s, `location`=%s,
            `follower_count`=%s, `following_count`=%s, `post_count`=%s
            """
            
            now = datetime.now()
            values = (
                user_data.get('id', ''),
                user_data.get('nickname', ''),
                user_data.get('avatar', ''),
                user_data.get('gender', ''),
                user_data.get('location', ''),
                user_data.get('follower_count', 0),
                user_data.get('following_count', 0),
                user_data.get('post_count', 0),
                now,
                # 更新值
                user_data.get('nickname', ''),
                user_data.get('avatar', ''),
                user_data.get('gender', ''),
                user_data.get('location', ''),
                user_data.get('follower_count', 0),
                user_data.get('following_count', 0),
                user_data.get('post_count', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存用户数据成功: {user_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存用户数据失败: {e}")
            self.conn.rollback()
            return False
    
    def save_activity(self, activity_data):
        """保存运动记录"""
        try:
            if not activity_data or not activity_data.get('id'):
                return False
            
            # 确保用户存在
            user_id = activity_data.get('user', {}).get('id')
            if user_id:
                self.cursor.execute("SELECT 1 FROM `users` WHERE `user_id` = %s", (user_id,))
                if not self.cursor.fetchone():
                    self.cursor.execute("""
                    INSERT INTO `users` (`user_id`, `nickname`, `created_at`)
                    VALUES (%s, %s, %s)
                    """, (user_id, activity_data.get('user', {}).get('nickname', ''), datetime.now()))
                    self.conn.commit()
            
            sql = """
            INSERT INTO `activities` 
            (`activity_id`, `user_id`, `type`, `title`, `description`, `distance`,
             `duration`, `calorie`, `start_time`, `likes_count`, `comments_count`,
             `route_data`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `likes_count`=%s, `comments_count`=%s
            """
            
            now = datetime.now()
            
            # 处理时间戳
            try:
                start_time = datetime.fromtimestamp(activity_data.get('start_time', 0))
            except:
                start_time = now
            
            # 处理JSON数据
            try:
                route_data = json.dumps(activity_data.get('route_data', {}))
            except:
                route_data = '{}'
            
            values = (
                activity_data.get('id', ''),
                user_id,
                activity_data.get('type', ''),
                activity_data.get('title', ''),
                activity_data.get('description', ''),
                activity_data.get('distance', 0),
                activity_data.get('duration', 0),
                activity_data.get('calorie', 0),
                start_time,
                activity_data.get('likes_count', 0),
                activity_data.get('comments_count', 0),
                route_data,
                now,
                # 更新值
                activity_data.get('likes_count', 0),
                activity_data.get('comments_count', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存运动记录成功: {activity_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存运动记录失败: {e}")
            self.conn.rollback()
            return False
    
    def save_route(self, route_data):
        """保存路线数据"""
        try:
            if not route_data or not route_data.get('id'):
                return False
            
            sql = """
            INSERT INTO `routes` 
            (`route_id`, `title`, `description`, `distance`, `author_id`,
             `likes_count`, `used_count`, `location`, `created_at`)
            VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s)
            ON DUPLICATE KEY UPDATE 
            `title`=%s, `description`=%s, `likes_count`=%s, `used_count`=%s
            """
            
            now = datetime.now()
            values = (
                route_data.get('id', ''),
                route_data.get('title', ''),
                route_data.get('description', ''),
                route_data.get('distance', 0),
                route_data.get('author', {}).get('id', ''),
                route_data.get('likes_count', 0),
                route_data.get('used_count', 0),
                route_data.get('location', ''),
                now,
                # 更新值
                route_data.get('title', ''),
                route_data.get('description', ''),
                route_data.get('likes_count', 0),
                route_data.get('used_count', 0)
            )
            
            self.cursor.execute(sql, values)
            self.conn.commit()
            logger.info(f"保存路线数据成功: {route_data.get('id')}")
            return True
        except Exception as e:
            logger.error(f"保存路线数据失败: {e}")
            self.conn.rollback()
            return False
    
    def random_sleep(self, min_sleep=1, max_sleep=3):
        """随机休眠，避免请求过快"""
        sleep_time = random.uniform(min_sleep, max_sleep)
        time.sleep(sleep_time)
    
    def run(self, max_pages=3, use_mock_data=True):
        """运行爬虫程序
        
        参数:
            max_pages: 爬取的最大页数
            use_mock_data: 是否使用模拟数据（当API请求失败时）
        """
        if not self.init_database():
            logger.error("数据库初始化失败，爬虫程序终止")
            return
        
        try:
            # 爬取热门动态
            logger.info("开始爬取热门运动动态...")
            for page in range(1, max_pages + 1):
                activities = self.fetch_popular_activities(page=page)
                logger.info(f"第{page}页，获取到{len(activities)}条动态")
                
                for activity in activities:
                    # 保存用户信息
                    user_data = activity.get('user')
                    if user_data and user_data.get('id'):
                        user_detail = self.fetch_user_info(user_data.get('id'))
                        if user_detail:
                            self.save_user(user_detail)
                    
                    # 保存活动记录
                    self.save_activity(activity)
                    self.random_sleep()
                
                self.random_sleep(2, 5)
            
            # 爬取热门路线
            logger.info("开始爬取热门路线...")
            for page in range(1, max_pages + 1):
                routes = self.fetch_routes(page=page)
                logger.info(f"第{page}页，获取到{len(routes)}条路线")
                
                for route in routes:
                    # 保存用户信息
                    author_data = route.get('author')
                    if author_data and author_data.get('id'):
                        user_detail = self.fetch_user_info(author_data.get('id'))
                        if user_detail:
                            self.save_user(user_detail)
                    
                    # 保存路线数据
                    self.save_route(route)
                    self.random_sleep()
                
                self.random_sleep(2, 5)
            
            logger.info("爬虫程序执行完成!")
        
        except Exception as e:
            logger.error(f"爬虫程序执行过程中出错: {e}")
        
        finally:
            # 关闭数据库连接
            if self.cursor:
                self.cursor.close()
            if self.conn:
                self.conn.close()

if __name__ == "__main__":
    crawler = CodoonCrawler()
    # 使用模拟数据运行
    crawler.run(max_pages=3, use_mock_data=True) 